#!/bin/sh
# set -x
#
@copyright_hash@
#
# hpclink -- link application with libhpcrun, libmonitor, PAPI and
# Xed2 statically by editing the compile line.
#
# Usage: hpclink [options] compiler file ...
#
#     -h, --help
#     --memleak
#     -u, --undefined  <symbol>
#     -v, --verbose
#     -V, --version
#
#  where <symbol> is a symbol name passed to the linker (may be used
#  multiple times).
#

#------------------------------------------------------------
# Values from configure
#------------------------------------------------------------

@launch_script_vars@

# Relative paths are relative to HPCTOOLKIT.
ext_libs_dir='@hpc_ext_libs_dir@'
hpcfnbounds_dir='libexec/hpctoolkit'
libhpcrun_dir='lib/hpctoolkit'
libmonitor_dir='@LIBMONITOR_RUN_DIR@'

# Absolute path or empty.
papi_dir='@OPT_PAPI_LIBPATH@'
papi_extra_libs='@papi_extra_libs@'

upc_libs='@OPT_UPC_LDFLAGS@'

hpctk_wrap_names='@LIBMONITOR_WRAP_NAMES@  @hpclink_extra_wrap_names@'

extra_wrap_names=
extra_hpc_files=
# Compiler and flags for array of fnbounds addresses.
CC='@CC@ @CFLAGS@'

hello="_hpc_hello_$$"
nm_addrs="_hpc_nm_addrs_$$"
cmd_out="_hpc_output_$$"

# Space-separated list of symbol names to force undefined.
undef_names=

# Space-separated list of libs (without -l) not to repeat on command line.
no_repeat_list='hugetlbfs'

prog_name=hpclink

#------------------------------------------------------------

cleanup()
{
    rm -f _hpc_*$$*
}

die()
{
    echo "hpclink: error: $*" 1>&2
    cleanup
    exit 1
}

mesg()
{
    echo "hpclink: $*"
}

unable_to_link()
{
    mesg "unable to link"
    echo
    echo "$command" $command_line_args
    echo
    cat "$cmd_out"
    echo
    die "unable to link"
}

#------------------------------------------------------------
# The darshan I/O characterization library
# (http://www.mcs.anl.gov/research/projects/darshan) is linked into
# applications by default on several DOE supercomputers. Darshan
# intercepts file operations performed within signal handlers while
# profiling, which can cause deadlock.  Refuse to link HPCToolkit 
# into applications if darshan will be included.
#------------------------------------------------------------
check_darshan()
{
    darshan_msg='Darshan is incompatible with HPCToolkit.
Unload darshan from your environment before using hpclink.'

    echo "x$PE_PKGCONFIG_LIBS" | grep -i darshan >/dev/null 2>&1
    if test $? -eq 0 ; then
        if test "$1" = "die" ; then
           die "$darshan_msg"
        else
           echo ; echo "warning: $darshan_msg"
        fi
    fi
}

#------------------------------------------------------------
# Usage Message
#------------------------------------------------------------

usage()
{
    cat <<EOF
Usage: hpclink [options] <link-command>

hpclink links HPCToolkit's performance measurement library into a
statically linked application.  (hpcrun's method for injecting its library
into a dynamically linked application will not work with a statically
linked applications.)

To link with hpclink, supply your application's normal link line as
<link-command>, which typically has the following form:
  <compiler> [link-options] <object-files> <libraries>

To control HPCToolkit's performance measurement library during an
application's execution, use the following environment variables:
  HPCRUN_EVENT_LIST=<event1>[@<period1>];...;<eventN>[@<periodN>]
                             : Sampling event list; hpcrun -e/--event
  HPCRUN_TRACE=1             : Enable tracing; hpcrun -t/--trace
  HPCRUN_PROCESS_FRACTION=<f>: Measure only a fraction <f> of the execution's
                               processes; hpcrun -f/-fp/--process-fraction
  HPCRUN_OUT_PATH=<outpath>  : Set output directory; hpcrun -o/--output

Options: Informational
  -v, --verbose        Verbose. Displays the original and modified command
                       lines.
  -V, --version        Print version information.
  -h, --help           Print help.

Options: Linking
  -dw, --double-wrap   Double quote the linker wrap options (-Wl,-Wl instead
                       of -Wl).  Used to work around problems with the 
                       Berkeley UPC compiler script (rarely needed).

  -fe <dir>, --front-end <dir>
                       In a cross-compile link, use the current install tree
                       for the back-end libraries and use <dir> for the install
                       directory (prefix) of the front-end tools (hpcfnbounds).

  --io                 Include the IO bytes read and written library.

  --ga                 Include GA (Global Arrays) wrappers.

  --memleak            Include HPCToolkit's memory leak detection libraries.

  --plugin name        Add the libraries and wrapped symbols specified by
                       the plugin 'name' in the plugins directory.

  -u <symbol>, --undefined <symbol>
                       Pass <symbol> to the linker as an undefined symbol.
                       This is to force the linker to pull in a reference
                       for that symbol.  May be used multiple times.
EOF
    check_darshan "mesg"
    exit 0
}

#------------------------------------------------------------
# Find path to this script
#------------------------------------------------------------

hpc_path_to_root=..
@export_hpctoolkit@

# Relative paths are relative to HPCTOOLKIT.
case "$ext_libs_dir" in
    /* ) ;;
    * )  ext_libs_dir="${HPCTOOLKIT}/${ext_libs_dir}" ;;
esac
case "$libhpcrun_dir" in
    /* ) ;;
    * )  libhpcrun_dir="${HPCTOOLKIT}/${libhpcrun_dir}" ;;
esac
case "$libmonitor_dir" in
    /* ) ;;
    * )  libmonitor_dir="${HPCTOOLKIT}/${libmonitor_dir}" ;;
esac

LD_LIBRARY_PATH="${ext_libs_dir}:${LD_LIBRARY_PATH}"
export LD_LIBRARY_PATH

#------------------------------------------------------------
# Step 1 -- Parse linker options, our options first.
#------------------------------------------------------------

@launch_early_options@

double_wrap=no
front_end_dir=no
my_plugin_list=
verbose=no
verb=:

while test "x$1" != x
do
    case "$1" in
	-h | --help )
	    usage
	    ;;
	-dw | --double-wrap )
	    double_wrap=yes
	    shift
	    ;;
	-fe | -front-end | --front-end )
	    test "x$2" != x || die "missing argument for --front-end"
	    front_end_dir="$2"
	    shift ; shift
	    ;;
	-io | --io )
	    io_wrap="${libhpcrun_dir}/libhpcrun_io_wrap.a"
	    test -f "$io_wrap" || die "unable to find: $io_wrap"
	    extra_hpc_files="$extra_hpc_files $io_wrap"
	    extra_wrap_names="$extra_wrap_names read write fread fwrite"
	    undef_names="$undef_names fwrite"
	    shift
	    ;;
	-ga | --ga )
	    ga_wrap="${libhpcrun_dir}/libhpcrun_ga_wrap.a"
	    test -f "$ga_wrap" || die "unable to find: $ga_wrap"
	    extra_hpc_files="$extra_hpc_files $ga_wrap"
	    extra_wrap_names="$extra_wrap_names \
                pnga_create pnga_create_handle \
                pnga_get pnga_put pnga_acc \
                pnga_nbget pnga_nbput pnga_nbacc pnga_nbwait \
                pnga_brdcst pnga_gop pnga_sync"
	    undef_names="$undef_names pnga_create"
	    shift
	    ;;
	-memleak | --memleak )
	    memleak_wrap="${libhpcrun_dir}/libhpcrun_memleak_wrap.a"
	    test -f "$memleak_wrap" || die "unable to find: $memleak_wrap"
	    extra_hpc_files="$extra_hpc_files $memleak_wrap"
	    extra_wrap_names="$extra_wrap_names posix_memalign memalign valloc"
	    extra_wrap_names="$extra_wrap_names calloc free malloc realloc"
	    undef_names="$undef_names malloc"
	    shift
	    ;;
	-plugin | --plugin )
	    test "x$2" != x || die "missing argument for --plugin"
	    my_plugin_list="${my_plugin_list} $2"
	    shift ; shift
	    ;;
	-u | --undefined )
	    test "x$2" != x || die "missing argument for -u"
	    undef_names="${undef_names} $2"
	    shift ; shift
	    ;;
	-v | --verbose )
	    verbose=yes
	    verb=mesg
	    shift
	    ;;
	-- )
	    shift
	    break
	    ;;
	-* )
	    die "unknown option: $1"
	    ;;
	* )
	    break
	    ;;
    esac
done

check_darshan "die"

#
# In a cross-compile link, we use fnbounds from the front-end tree and
# hpcrun libs, externals libs, PAPI and CC from the back-end tree.
# Note: since fnbounds is a program with a launch script, we need to
# reset HPCTOOLKIT to the front-end root before running fnbounds.
#
if test "$front_end_dir" != no ; then
    HPCTOOLKIT=`cd "$front_end_dir" && pwd`
    if test ! -d "$HPCTOOLKIT" ; then
        die "bad front-end directory: $front_end_dir"
    fi
fi

hpcfnbounds_dir="${HPCTOOLKIT}/${hpcfnbounds_dir}"
hpcfnbounds="${hpcfnbounds_dir}/hpcfnbounds"
test -x "$hpcfnbounds" || die "missing hpcfnbounds: $hpcfnbounds"

#
# Add plugin files: hpclink --plugin name
# Plugin name and hpclink_files can be absolute path or relative to
# the plugins directory.
#
extra_plugin_files=
for plugin in $my_plugin_list
do
    $verb "plugin: $plugin"
    case "$plugin" in
	/* ) conf_file="$plugin" ;;
	* )  conf_file="${libhpcrun_dir}/plugins/${plugin}" ;;
    esac
    my_plugin_dir=`dirname "$conf_file"`
    test -f "$conf_file" || die "missing plugin config file: $conf_file"
    file "$conf_file" | grep text >/dev/null 2>&1
    test $? -eq 0 || die "bad plugin config file: $conf_file"
    hpclink_files=
    hpclink_wrap_names=
    hpclink_undefined_names=
    . "$conf_file"
    for file in $hpclink_files ; do
	case "$file" in
	    /* ) abs_file="$file" ;;
	    * )  abs_file="${my_plugin_dir}/${file}" ;;
	esac
	test -f "$abs_file" || die "missing plugin file: $abs_file"
	$verb "plugin file: $abs_file"
	extra_plugin_files="$extra_plugin_files $abs_file"
    done
    $verb "plugin wrap names: $hpclink_wrap_names"
    $verb "plugin undefined names: $hpclink_undefined_names"
    extra_wrap_names="$extra_wrap_names $hpclink_wrap_names"
    undef_names="$undef_names $hpclink_undefined_names"
done

#
# Must have a compiler command and at least one argument.
#
test "x$2" != x || usage
command="$1"
shift

#
# Read the command line for: -l<lib> and -o <file> arguments.
# It's important not to change the command line here.
#
appl_libs=
appl_out=a.out
prev_arg=no
for arg in "$@"
do
    if test "x$prev_arg" = x-o ; then
	appl_out="$arg"
    else
	case "$arg" in
	    -l?* )
		copy=yes
		for lib in $no_repeat_list ; do
		    if test "x$arg" = "x-l$lib" ; then
			copy=no
			break
		    fi
		done
		if test "$copy" = yes ; then
		    appl_libs="$appl_libs $arg"
		fi
		;;
	esac
    fi
    prev_arg="$arg"
done

#------------------------------------------------------------
# Step 2 -- Build the new compile line.
#------------------------------------------------------------

# New compile line:
# undef-args, wrap-args, $@, libhpcrun.o, nm-addrs, libhpcrun_wrap.a,
# libmonitor, plugin libraries, PAPI, UPC, Xed2, -lpthread, -ldl,
# repeat-application-libs.

if test "$verbose" = yes ; then
    mesg "original command line:"
    echo "$command $@"
    echo
    mesg "wrapped symbols:"
    echo $hpctk_wrap_names $extra_wrap_names
    echo
    mesg "initial undefined symbols:"
    echo $undef_names
    echo
fi

# Work around a problem with the Berkeley UPC compiler script.
if test "$double_wrap" = yes ; then
    linker_prefix=-Wl,-Wl
else
    linker_prefix=-Wl
fi

undef_args=
for name in $undef_names
do
    undef_args="${undef_args} ${linker_prefix},--undefined=${name}"
done

wrap_args=
for name in $hpctk_wrap_names $extra_wrap_names
do
    wrap_args="${wrap_args} ${linker_prefix},--wrap=${name}"
done

libhpcrun="${libhpcrun_dir}/libhpcrun.o"
test -f "$libhpcrun" || die "no such file: $libhpcrun"

# FIXME: add LZMA static library at the end of libhpcrun.o
# Ideally the library is included in libhpcrun.o, but somwehow not all
# routines are included
liblzma="${ext_libs_dir}/liblzma.a"
set -- $undef_args $wrap_args $libhpcrun "$liblzma"  "$@"
set -- "$@" "${nm_addrs}.o"

libhpcrun_wrap="${libhpcrun_dir}/libhpcrun_wrap.a"
test -f "$libhpcrun_wrap" || die "no such file: $libhpcrun_wrap"
set -- "$@" "$libhpcrun_wrap" $extra_hpc_files

libmonitor="${libmonitor_dir}/libmonitor_wrap.a"
test -f "$libmonitor" || die "no such file: $libmonitor"
set -- "$@" "$libmonitor" $extra_plugin_files

if test -d "$papi_dir" ; then
    set -- "$@" "-L${papi_dir}" -lpapi $papi_extra_libs
fi

set -- "$@" $upc_libs

# FIXME: shouldn't always need pthread, and shoudn't ever need dl.
set -- "$@" -ldl -lrt -lpthread
set -- "$@" $appl_libs

#------------------------------------------------------------
# Step 3 -- Link with dummy nm file.
#------------------------------------------------------------

rm -f "${hello}.c" "$hello"
cat <<EOF > "${hello}.c"
int main(int argc, char **argv)
{
    return (0);
}
EOF
$CC -o "$hello" "${hello}.c"

rm -f "${nm_addrs}.c" "${nm_addrs}.o"
$hpcfnbounds -c "$hello" > "${nm_addrs}.c" || die "hpcfnbounds failed on $hello"
$CC -c -o "${nm_addrs}.o" "${nm_addrs}.c"

#
# Some compiler scripts add extra function calls to the end of the
# link line that fail to trigger --wrap.  So, if the first compile
# fails with undefined references to __wrap_foo, then add those names
# to the undef list and try again.  Do the same for an undefined
# reference to 'main' (gfortran does some bad things).
#
num=1
success=no
extra_undef_opts=
while :
do
    mesg "preliminary link $num ..."
    if test "$verbose" = yes ; then
	echo "$command $@"
	echo
    fi

    rm -f "$cmd_out"
    command_line_args="$*"
    $command "$@" >"$cmd_out" 2>&1
    if test $? -eq 0 && test -f "$appl_out" ; then
	success=yes
	break
    fi

    # If the link fails, parse the compiler output for 'undefined
    # references' and add to the next link.
    new_undef_list=
    exec <"$cmd_out"
    while read line
    do
        name=
        if echo "$line" | grep -E -i -e 'undef.*__wrap_' >/dev/null 2>&1 ; then
	    name=`expr "$line" : '.*\(__wrap_[A-Za-z0-9_]*\)'`
	elif echo "$line" | grep -E -i -e 'undef.*main' >/dev/null 2>&1 ; then
	    name=main
	fi
	if test "x$name" != x ; then
	    echo "$extra_undef_opts" | grep -e " $name " >/dev/null 2>&1
	    if test $? -ne 0 ; then
		set -- "${linker_prefix},--undefined=${name}" "$@"
		extra_undef_opts="$extra_undef_opts -u $name "
		new_undef_list="$new_undef_list $name"
	    fi
	fi
    done
    if test "$verbose" = yes ; then
	mesg "new undefined symbols: $new_undef_list"
	echo
    fi

    # quit when there are no new undefined references
    if test "x$new_undef_list" = x ; then
	break
    fi

    num=`expr $num + 1`
done

if test "$success" != yes ; then
    unable_to_link
fi

# Require the output to be statically linked.
if file "$appl_out" | grep -i static >/dev/null ; then :
else
    mv -f "$appl_out" "${appl_out}.failed"
    die "program not statically linked (maybe forgot -static): $appl_out"
fi

#------------------------------------------------------------
# Step 4 -- Link with real nm file.
#------------------------------------------------------------

mesg "final link ..."
if test "$verbose" = yes ; then
    echo "$command $@"
    echo
fi

rm -f "${nm_addrs}.c" "${nm_addrs}.o"
$hpcfnbounds -c "$appl_out" > "${nm_addrs}.c" || die "hpcfnbounds failed on $appl_out"
$CC -c -o "${nm_addrs}.o" "${nm_addrs}.c"

rm -f "$cmd_out"
command_line_args="$*"
$command "$@" >"$cmd_out" 2>&1
if test $? -ne 0 || test ! -f "$appl_out" ; then
    unable_to_link
fi
mesg "done"

if test "x$extra_undef_opts" != x ; then
    mesg "to reduce the number of preliminary links, add these options to hpclink:"
    echo hpclink $extra_undef_opts $command ...
fi

cleanup
exit 0
